import os,sys
sys.path.append(r"D:\\source\\work\\spiders")

from zgjw_zt.db import *
from zgjw_zt.spider import spider
from zgjw_zt.utils import *
from urllib import parse
import time
import datetime
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
import shutil
import traceback

ztspider = spider('http://www.81.cn/xue-xi/102726.htm')

def run(index):
    num = 2
    while True:
        model,left = db.popurl()
        if num <= 0:
            break
        if model == None:
            num = num - 1
            print('线程{}:空'.format(index))
            time.sleep(2)
            continue
        print('线程{},{}待处理:{}'.format(index,left,model.url))
        num = 5
        try:
            if model.filetype == 5:
                ztspider.node(model)
                pass
            elif model.filetype == 1:  # 网页
                if model.url.find('node_') > -1:
                    pass
                    ztspider.node(model)
                elif model.url.find('tv') > -1:
                    ztspider.tv(model)
                    pass
                elif model.url.find('content_') > -1:
                    ztspider.content(model)
                    pass
                else:
                    ztspider.content(model)
            elif model.filetype == 2:  # 网页资源
                if model.url.find('.css') > -1:
                    ztspider.css(model)
                    pass
                else:
                    downfile(model.url, model.savepath)
                    pass
            elif model.filetype == 3:  # 音视频资源
                if model.url.find('http') > -1:
                    downfile(model.url, model.savepath)
                else:
                    down_m3u8(model.url, model.savepath)

            if os.path.isfile(model.savepath):
                db.setstatus(model.id, 1)
            else:
                db.setstatus(model.id, 10)
        except:
            with open('error.txt', 'a+') as f:
                f.write('{}\r'.format(model.url))
            print('线程{}:{}'.format(index, model.url))
            db.setstatus(model.id, 10)

def downdata():
    basedir = r'\\192.168.10.60\d$\chuanshu\数据采集[张俊峰]\xxqj'
    db = urls()
    db.create_table()
    ztspider.index()
    #model = db.get(id=23774)
    #ztspider.node(model)
    #downfile(model.url,model.savepath)
    #model = db.get(url='http://photo.81.cn/pla/2018-12/28/content_9389560_10.htm')
    #ztspider.content(model)
    #model = db.get(id=3)
    #ztspider.node(model)
    #ztspider.parse_rili()
    #exit()
    threadnum = 50
    p = ThreadPoolExecutor(max_workers=threadnum)
    obj_l = []
    for i in range(threadnum):
        obj = p.submit(run, i)
        obj_l.append(obj)
    p.shutdown()  # 等同于p.close(),p.join()

    shutil.copy('data.db','xxqj/data.db')
    dt = datetime.datetime.now()
    print('[{}]下载完毕,开始迁移数据'.format(dt.strftime('%m-%d %H:%M:%S')))

    filename = 'xxqj'+str(datetime.datetime.now().strftime('%m%d%H%M'))
    shutil.make_archive(filename,'zip',root_dir='xxqj')
    shutil.rmtree('xxqj')
    filename = filename + '.zip'
    destpath = os.path.join(basedir,filename)
    shutil.move(filename,destpath)
    print('[{}]数据迁移完毕'.format(dt.strftime('%m-%d %H:%M:%S')))

def main():
    try:
        t = threading.Thread(target=downdata)
        t.setDaemon(True)
        t.start()
        t.join(60 * 60)
    except:
        traceback.print_exc()
    while True:
        n = 2*60*60
        while n>0:
            h = int(n/3600)
            m = int(n%3600/60)
            s = n%60
            lefttime = '{}小时{}分钟{}秒'.format(h,m,s)
            print('离下次更新还剩{}'.format(lefttime))
            time.sleep(1)
            n = n-1

if __name__ == '__main__':
    main()